use strict;
use warnings;
use threads;
use threads::shared;
use DB_File;

my $site = "docs.jquery.com";
my $dir = "d:/temp/jquery";

my %urls;
tie %urls, "DB_File", "d:/temp/jquerydb" or die $!;

$urls{"http://" . $site . "/"} = 0;

my @queue : shared;
my @new : shared;
my @finished : shared;
my @threads;

for (1 .. 10) {
	my $thr = threads->new(\&threadfunc);
	push @threads, $thr;
}

while (1) {
	{
		lock @queue;
		for my $url (keys %urls) {
			if ($urls{$url} == 0) {
				push @queue, $url;
				$urls{$url} = 1;
			}
		}
	}
	{
		lock @new;
		while (my $new = shift @new) {
			if (!defined($urls{$new})) {
				$urls{$new} = 0;
			}
		}
	}
	{
		lock @finished;
		while (my $finished = shift @finished) {
			if (defined($urls{$finished})) {
				$urls{$finished} = 2;
			}
		}
	}
	my $goon = 0;
	for my $url (keys %urls) {
		if ($urls{$url} <= 1) {
			$goon = 1;
		}
	}
	last unless $goon;
	sleep(1);
}

for my $thr (@threads) {
	$thr->detach;
}

untie %urls;

exit 0;

sub threadfunc {
	require LWP::UserAgent;
	my $ua = LWP::UserAgent->new;
	$ua->default_header('Accept' => 'text/html,text/plain');
	while (1) {
		my $url;
		{
			lock @queue;
			$url = shift @queue;
		}
		if ($url) {
			print threads->tid, "\tgot $url\n";
			my $res = $ua->get($url);
			my $content = $res->content;
			my @news = $content =~ m|<a[^>]+href=(?<quote>["']?)(\S+?)\k<quote>>|ig;
			print join("\n", @news), "\n";
			{
				lock @finished;
				push @finished, $url;
			}
		}
		sleep(1);
	}
}